import re #regular expression library
import urllib #url library
import string #string functions
import sys    #system library

#===============================================================================
# getUrlData - Get the data from URL to a FILE
#===============================================================================
def getUrlData(url):
    #TO-DO check connection
    urlSearch = url
    urlfile = urllib.urlopen(urlSearch)
    
    #urldata will contain the url data with out null chars
    urldata = urlfile.read().replace("\0","")
    urlfile.close()
    return urldata


#===============================================================================
# extractResultsLive - Extract the urls from Live search page
#===============================================================================
def extractResultsLive(urldata):
    
    textQueryGetResultBlock = "\<div id=\"results\"\>.+?\</div\>"
    resultBlock = re.findall(textQueryGetResultBlock, urldata, re.DOTALL)
    resultBlockAsString = ''.join(resultBlock)
    
    textQueryGetResultsUrls = "(?<=\<h3\>\<a href=\").+?(?=\")"
    resultsUrls = re.findall(textQueryGetResultsUrls, resultBlockAsString, re.DOTALL)

    for i in range(0, len(resultsUrls)):
        resultsUrls[i] = resultsUrls[i].rstrip('/') #strips the '/' if there

    return resultsUrls
#end extractResultsLive    
        
#===============================================================================
# extractResultsYahoo - Extract the urls from Yahoo search page
#===============================================================================
def extractResultsYahoo(urldata):  
    textQueryGetResultBlock = "\<h2\>WEB RESULTS\</h2\>.+?\</ol\>"
    resultBlock = re.findall(textQueryGetResultBlock, urldata, re.DOTALL)
    resultBlockAsString = ''.join(resultBlock)

    textQueryGetResultsUrls = "(?<=\<span class=url\>).+?(?=\</span\>)"
    resultsUrls = re.findall(textQueryGetResultsUrls, resultBlockAsString, re.DOTALL)
    
    for i in range(0, len(resultsUrls)):  #get rid of bold tag and add http://
        resultsUrls[i] = resultsUrls[i].replace("<b>", "").replace("</b>", "")
        resultsUrls[i] = "http://" + resultsUrls[i]
        resultsUrls[i] = resultsUrls[i].rstrip('/')  #strips the '/' if there

    return resultsUrls
#end extractResultsYahoo

#===============================================================================
# getNextUrlLive - Get Live search page and return the URL of the next search page
#===============================================================================
def getNextUrlLive(urldata):   
    textQueryGetNextUrl = "(?<=class\=\"nP\" href\=\").+?(?=\"\>Next)"    
    nextUrl = re.findall(textQueryGetNextUrl, urldata, re.DOTALL)  
    if len(nextUrl) != 0: 
        fixNextUrl = string.replace(nextUrl[0], ";", "")
        fixNextUrl = string.replace(fixNextUrl, "amp", "")
        fixNextUrl = "http://search.live.com" + fixNextUrl 
        return fixNextUrl
    else:
        return ""
#end getNextUrlYahoo

#===============================================================================
# getNextUrlYahoo - Get Yahoo search page and return the URL of the next search page 
#===============================================================================
def getNextUrlYahoo(urldata):
    textQueryGetNextUrl = "(?<=id\=\"pg-next\" class\=\"pg\" href\=\").+?(?=\"\>Next)"
    nextUrl = re.findall(textQueryGetNextUrl, urldata, re.DOTALL)
    if len(nextUrl) != 0:
        return nextUrl[0]
    else:
        return ""
#end getNextUrlYahoo

#===============================================================================
# seperateMatches - Get the previous results, matches, and current results
# Seperate the current results between the prev results and the matches lists
#===============================================================================
def seperateMatches(prev, matches, results):
    for result in results:
        if result in prev:
            prev.remove(result)
            matches.append(result)
            print "<p>%d : <a href=%s>%s</a></p>" % (len(matches), result, result)
        else:
            prev.append(result)
#end seperateMatches

#===============================================================================
#************************************ MAIN *************************************
#===============================================================================

#Get the values from the command line
textQuery = sys.argv[1]
targetMatches = int(sys.argv[2])   #Cast to integer

query = urllib.quote(textQuery) #replaces unsafe and special chars with %ascii 

#Defines lists for results and matches
results = []
matchesFound = []


liveData = getUrlData("http://search.live.com/results.aspx?q=" + query)
firstResultsLive = extractResultsLive(liveData)  
#print "Live:"
#print firstResultsLive

yahooData = getUrlData("http://search.yahoo.com/search?p=" + query)
firstResultsYahoo = extractResultsYahoo(yahooData)
#print "Yahoo:"
#print firstResultsYahoo

seperateMatches(results, matchesFound, firstResultsLive)
seperateMatches(results, matchesFound, firstResultsYahoo)

yahooNext = getNextUrlYahoo(yahooData)
liveNext = getNextUrlLive(liveData)


while (len(matchesFound) < targetMatches) and (yahooNext != "" or liveNext != ""):
    if yahooNext != "":
        yahooData = getUrlData(yahooNext)
        resultsYahoo = extractResultsYahoo(yahooData)
        seperateMatches(results, matchesFound, resultsYahoo)
        yahooNext = getNextUrlYahoo(yahooData)
    
    if liveNext != "":
        liveData = getUrlData(liveNext)
        resultsLive = extractResultsLive(liveData)
        seperateMatches(results, matchesFound, resultsLive)
        liveNext = getNextUrlLive(liveData)

print
print "<b>finished!</b>"
#end MAIN    